In [6]:

    

from __future__ import print_function
import requests
import json
import pprint
import matplotlib.pyplot as plt
import traceback
import logging
import numpy
import time
import os
import random


    

In [2]:

    

def request_grid(x0, x1, y0, y1, logger_func=None):
    '''Request wells inside box.
    
    Args:
      x0, x1 (float): longitudes
      y0, y1 (float): latitudes (include negative sign for southern hemisphere)
      
    Returns: list of dicts with well data. Note max list size is 10K, so
    if the list is < 10K the search was exhaustive, if list == 10K then 
    you need to request smaller regions within.
    
    '''
    if logger_func is None:
        logger_func = lambda msg: None
    url_grid = ("https://www.waterconnect.sa.gov.au/_layouts/dfw.sharepoint.wdd/"
                "WDDDMS.ashx/GetGridData?Box={y0},{x0},{y1},{x1}")
    url = url_grid.format(x0=x0, x1=x1, y0=y0, y1=y1)
    r = requests.get(url)
    assert r.status_code == 200
    try:
        rdl = json.loads(r.text)
    except:
        if not logger_func is None:
            logger_func(traceback.format_exc().splitlines()[-1])
    logger_func("Found %s records" % (len(rdl), ))
    return rdl


    

In [3]:

    

for x in numpy.arange(138.5, 138.75, 0.125):
    for y in numpy.arange(-35, -34.75, 0.125):
        x0 = x
        x1 = x + 0.125
        y0 = y
        y1 = y + 0.125
        print("Looking at: %s\t%s\t%s\t%s" % (x0, x1, y0, y1))
        fn = os.path.join("downloads", "gd_wc_dl_{x0}_{x1}_{y0}_{y1}.json".format(x0=x0, x1=x1, y0=y0, y1=y1))
        print("Writing results to %s" % (fn, ))
        result = request_grid(x0, x1, y0, y1, logger_func=print)
        with open(fn, mode="w") as f:
            f.write(json.dumps(result))
        sleep_secs = random.randint(10, 90)
        print("Waiting %s seconds" % (sleep_secs, ))
        time.sleep(sleep_secs)


    

    




Looking at: 138.5	138.625	-35.0	-34.875
Writing results to gd_wc_dl_138.5_138.625_-35.0_-34.875.json
Found 6888 records
Waiting 27 seconds
Looking at: 138.5	138.625	-34.875	-34.75





    




C:\Users\kent\Anaconda\lib\site-packages\requests\packages\urllib3\util\ssl_.py:90: InsecurePlatformWarning: A true SSLContext object is not available. This prevents urllib3 from configuring SSL appropriately and may cause certain SSL connections to fail. For more information, see https://urllib3.readthedocs.org/en/latest/security.html#insecureplatformwarning.
  InsecurePlatformWarning
C:\Users\kent\Anaconda\lib\site-packages\requests\packages\urllib3\util\ssl_.py:90: InsecurePlatformWarning: A true SSLContext object is not available. This prevents urllib3 from configuring SSL appropriately and may cause certain SSL connections to fail. For more information, see https://urllib3.readthedocs.org/en/latest/security.html#insecureplatformwarning.
  InsecurePlatformWarning






    




Writing results to gd_wc_dl_138.5_138.625_-34.875_-34.75.json
Found 3261 records
Waiting 24 seconds
Looking at: 138.625	138.75	-35.0	-34.875
Writing results to gd_wc_dl_138.625_138.75_-35.0_-34.875.json
Found 3130 records
Waiting 29 seconds
Looking at: 138.625	138.75	-34.875	-34.75





    




C:\Users\kent\Anaconda\lib\site-packages\requests\packages\urllib3\util\ssl_.py:90: InsecurePlatformWarning: A true SSLContext object is not available. This prevents urllib3 from configuring SSL appropriately and may cause certain SSL connections to fail. For more information, see https://urllib3.readthedocs.org/en/latest/security.html#insecureplatformwarning.
  InsecurePlatformWarning
C:\Users\kent\Anaconda\lib\site-packages\requests\packages\urllib3\util\ssl_.py:90: InsecurePlatformWarning: A true SSLContext object is not available. This prevents urllib3 from configuring SSL appropriately and may cause certain SSL connections to fail. For more information, see https://urllib3.readthedocs.org/en/latest/security.html#insecureplatformwarning.
  InsecurePlatformWarning






    




Writing results to gd_wc_dl_138.625_138.75_-34.875_-34.75.json
Found 2134 records
Waiting 40 seconds

In [9]:

    

fns1 = ["gd_wc_dl_138.5_138.625_-35.0_-34.875.json",
        "gd_wc_dl_138.5_138.625_-34.875_-34.75.json",
        "gd_wc_dl_138.625_138.75_-35.0_-34.875.json",
        "gd_wc_dl_138.625_138.75_-34.875_-34.75.json"]
fns = [os.path.join("downloads", fn) for fn in fns1]
alldata = []
for fn in fns:
    if os.path.isfile(fn):
        with open(fn, mode="r") as f:
            data = json.load(f)
            print("%s\t%s" % (len(data), fn))
            alldata += data
print(len(alldata))
allfn = os.path.join("downloads", "gd_wc_dl_138.5_138.75_-35.0_-34.75.json")
print("Writing results to %s" % (allfn, ))
with open(allfn, mode="w") as f:
    f.write(json.dumps(alldata))
print("done")


    

    




6888	gd_wc_dl_138.5_138.625_-35.0_-34.875.json
3261	gd_wc_dl_138.5_138.625_-34.875_-34.75.json
3130	gd_wc_dl_138.625_138.75_-35.0_-34.875.json
2134	gd_wc_dl_138.625_138.75_-34.875_-34.75.json
15413
Writing results to gd_wc_dl_138.5_138.75_-35.0_-34.75.json
done

In [ ]: